import json
import os
import time

import requests
from Tools.scripts.mkreal import join
from lxml import etree
import pymongo
import re
import threading
import multiprocessing
from fake_useragent import UserAgent
"""
使用了多线程
共爬取25336条数据
"""
mongo_conn = pymongo.MongoClient()
counter = 0
ua = UserAgent()


def thread_function(datas):
    while True:
        if len(datas) > 0:
            data = datas.pop(0)
            category_id = data[0]
            print(category_id)
            category_name = data[1]
            print(category_name)
            second_category_id = data[2]
            second_category_name = data[3]
            global counter
            page = 0
            while True:
                page += 1
                if page == 1:
                    url = f"https://www.xinpianchang.com/_next/data/7iD9y4Ok2pZK53ds5V-ny/discover/article/{category_id}-{second_category_id}.json"
                else:
                    url = f"https://www.xinpianchang.com/_next/data/7iD9y4Ok2pZK53ds5V-ny/discover/article/{category_id}-{second_category_id}-all-all-0-0-pick-pp{page}.json"
                headers = {
                    "Accept": "*/*",
                    "Accept-Language": "zh-CN,zh;q=0.9",
                    "Cache-Control": "no-cache",
                    "Connection": "keep-alive",
                    "Pragma": "no-cache",
                    "Referer": "https://www.xinpianchang.com/discover/article-1-355-all-all-0-0-pick",
                    "Sec-Fetch-Dest": "empty",
                    "Sec-Fetch-Mode": "cors",
                    "Sec-Fetch-Site": "same-origin",
                    "User-Agent": ua.random,
                    "sec-ch-ua": "\"Chromium\";v=\"130\", \"Google Chrome\";v=\"130\", \"Not?A_Brand\";v=\"99\"",
                    "sec-ch-ua-mobile": "?0",
                    "sec-ch-ua-platform": "\"Windows\"",
                    "x-nextjs-data": "1"
                }
                cookies = {
                    "Device_ID": "3i8fyc48lm216lhdg",
                    "MEIQIA_TRACK_ID": "2nBBjy6vGLmtVbTq8KcQfAmHVUA",
                    "MEIQIA_VISIT_ID": "2nBBjvM2aFubeWrmhoYqna0Gypn",
                    "_ga": "GA1.2.1447999405.1729768816",
                    "FECW": "31811a46fb46011dda3645e729a0b7a534e1d1ff8073d6011fb291767e13ef4671e5ebcbe8c3c2629e2db241a5132bc88793e7b51b9b5d09de16368583a5e971e1cabcee6b4391b0c90a1d5e14de17702e",
                    "Hm_lvt_446567e1546b322b726d54ed9b5ad346": "1729682423,1729841524,1729905400,1730340805",
                    "HMACCOUNT": "AFD53293F7A60F46",
                    "PHPSESSID": "adcjsdegvrdk42hno3j3cd8377",
                    "Authorization": "C13D3888F150E9C7EF150E4A52F150E8160F150E574DDCACE586",
                    "Hm_lpvt_446567e1546b322b726d54ed9b5ad346": "1730722122",
                    "sensorsdata2015jssdkcross": "%7B%22distinct_id%22%3A%2215041561%22%2C%22first_id%22%3A%22192f7137ed8678-00dec56d6b4e0a-26011951-2073600-192f7137ed988e%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_utm_source%22%3A%22xpcWeb%22%2C%22%24latest_utm_medium%22%3A%22navigatorCate%22%2C%22%24latest_utm_campaign%22%3A%22scjp%22%7D%2C%22%24device_id%22%3A%221926edb6166a-07d127846f2749-26001051-2073600-1926edb6167938%22%7D",
                    "FECA": "6UeXZP+0qkCL/eXe9QGTuX2OQyxzdjeSkdSVtoJ67kAU2tjH34RCaRKg+nAjU8t4J3YK6Uf9GBK7aw+79KXIi3jUUOf5lhqn9f9oqZwDUEVFbTlVzcqBZcWO7W0j6+pj82QdfcfNjruXI6DBUPJVwI9sumKJtTdtg1Xow8tqGbmfEqLxtHqpL9MvgGylIBaMse"
                }

                params = {
                    "FECU": "6UeXZP+0qkCL/eXe9QGTuX2OQyxzdjeSkdSVtoJ67kAU2tjH34RCaRKg+nAjU8t4J3YK6Uf9GBK7aw+79KXIi3jUUOf5lhqn9f9oqZwDUEVFbTlVzcqBZcWO7W0j6+pj82QdfcfNjruXI6DBUPJVwI0q+6u8i74l/Klx5aPIiBEROJv6BeQrWFu4I14nj2dnKe"
                }
                detail_response = requests.get(url, headers=headers, cookies=cookies, params=params)
                detail_result = detail_response.json()
                every_page_detail_list = detail_result['pageProps']['discoverArticleData']['list']
                for detail in every_page_detail_list:
                    counter += 1
                    try:
                        detail_author = f"{detail['author']['role']}+{detail['author']['userId']}"
                    except:
                        detail_author = "无作者"
                    try:
                        detail_title = detail['title']
                    except:
                        detail_title = "无标题"
                    try:
                        detail_url = detail['web_url']
                    except:
                        detail_url = "无链接"
                    count_collect = detail['count']['count_collect']
                    count_comment = detail['count']['count_comment']
                    count_like = detail['count']['count_like']
                    count_share = detail['count']['count_share']
                    count_view = detail['count']['count_view']
                    publish_time = detail['publish_time']
                    print(counter,page, category_id,category_name, second_category_id, second_category_name)
                    mongo_conn.xinpianchang.informations.insert_one({
                        "总数": counter,
                        "category": category_name,
                        "second_category": second_category_name,
                        "视频标题": detail_title,
                        "视频连接": detail_url,
                        "作者信息": detail_author,
                        "收藏数量": count_collect,
                        "评论数量": count_comment,
                        "点赞数量": count_like,
                        "分享数量": count_share,
                        "观看次数": count_view,
                        "发布时间": publish_time,
                    })
                time.sleep(1)
                if detail_result['pageProps']['discoverArticleData']['next_page_url'] == "":
                    break
        else:
            break


def main():
    print(f"开始爬取,进程号/线程号{multiprocessing.current_process().ident}/{threading.get_ident()}")
    datas = []
    headers = {
        "Accept": "*/*",
        "Accept-Language": "zh-CN,zh;q=0.9",
        "Cache-Control": "no-cache",
        "Connection": "keep-alive",
        "Pragma": "no-cache",
        "Referer": "https://www.xinpianchang.com/discover/article-16-18-all-all-0-0-score-pp2",
        "Sec-Fetch-Dest": "empty",
        "Sec-Fetch-Mode": "cors",
        "Sec-Fetch-Site": "same-origin",
        "User-Agent": ua.random,
        "sec-ch-ua": "\"Chromium\";v=\"130\", \"Google Chrome\";v=\"130\", \"Not?A_Brand\";v=\"99\"",
        "sec-ch-ua-mobile": "?0",
        "sec-ch-ua-platform": "\"Windows\""
    }
    cookies = {
        "Device_ID": "3i8fyc48lm216lhdg",
        "MEIQIA_TRACK_ID": "2nBBjy6vGLmtVbTq8KcQfAmHVUA",
        "MEIQIA_VISIT_ID": "2nBBjvM2aFubeWrmhoYqna0Gypn",
        "Authorization": "4EC46BA2681DB6A66681DB44C5681DBACB6681DB65EE9AFC6F64",
        "sensorsdata2015jssdkcross": "%7B%22distinct_id%22%3A%2215041561%22%2C%22first_id%22%3A%221926edb6166a-07d127846f2749-26001051-2073600-1926edb6167938%22%2C%22props%22%3A%7B%22%24latest_traffic_source_type%22%3A%22%E7%9B%B4%E6%8E%A5%E6%B5%81%E9%87%8F%22%2C%22%24latest_search_keyword%22%3A%22%E6%9C%AA%E5%8F%96%E5%88%B0%E5%80%BC_%E7%9B%B4%E6%8E%A5%E6%89%93%E5%BC%80%22%2C%22%24latest_referrer%22%3A%22%22%2C%22%24latest_utm_source%22%3A%22xpcWeb%22%2C%22%24latest_utm_medium%22%3A%22navigatorCate%22%2C%22%24latest_utm_campaign%22%3A%22scjp%22%7D%2C%22%24device_id%22%3A%221926edb6166a-07d127846f2749-26001051-2073600-1926edb6167938%22%7D",
        "_ga": "GA1.2.1447999405.1729768816",
        "FECW": "31811a46fb46011dda3645e729a0b7a534e1d1ff8073d6011fb291767e13ef4671e5ebcbe8c3c2629e2db241a5132bc88793e7b51b9b5d09de16368583a5e971e1cabcee6b4391b0c90a1d5e14de17702e",
        "Hm_lvt_446567e1546b322b726d54ed9b5ad346": "1729682423,1729841524,1729905400,1730340805",
        "HMACCOUNT": "AFD53293F7A60F46",
        "PHPSESSID": "adcjsdegvrdk42hno3j3cd8377",
        "Hm_lpvt_446567e1546b322b726d54ed9b5ad346": "1730442052",
        "FECA": "6UeXZP+0qkCL/eXe9QGTuX2OQyxzdjeSkdSVtoJ67kAXmAhK19SgCA/Udkw93IVrUgDGUjAiiTRuPf8uDEHuD52A6M/w40WRDxZ86UfwFS/vhv65lIwA2Yk1oJ4aq6FpV8ljmuk2fnis6Xr+WpPcQXWFs0z82Qh3zMtdfkadZPmQBfkh+qnFVt/CrN4yKGhc2e"
    }
    url = "https://www.xinpianchang.com/api/xpc/v2/article/category"
    params = {
        "FECU": "6UeXZP+0qkCL/eXe9QGTuX2OQyxzdjeSkdSVtoJ67kAXmAhK19SgCA/Udkw93IVrUgDGUjAiiTRuPf8uDEHuD52A6M/w40WRDxZ86UfwFS/vhv65lIwA2Yk1oJ4aq6FpV8ljmuk2fnis6Xr+WpPcQXREFvxfSbZQuKlJAebAPxjbEkoIvNu5QeowJqkrxklJde"
    }
    response = requests.get(url, headers=headers, cookies=cookies, params=params)
    result = response.content.decode()
    result = json.loads(result)
    for category in result["data"]["list"]:
        for second_category in category["children"]:
            datas.append([category['id'], category['name'], second_category['id'], second_category['name']])
    thread_list = []
    print(datas)
    for i in range(6):
        t = threading.Thread(target=thread_function, args=(datas,))
        thread_list.append(t)
        t.start()

    for t in thread_list:
        t.join()

    print('结束')


if __name__ == '__main__':
    main()
